{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 测试tokenzier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bos token: None\n",
      "bos id None\n",
      "eos token: <|im_end|>\n",
      "eos id 151645\n",
      "pad token <|endoftext|>\n",
      "pad_id: 151643\n",
      "vocab size 151643\n",
      "151646\n",
      "[151644, 8948, 198]\n",
      "<|im_start|>system\n",
      "You are a helpful assistant.<|im_end|>\n",
      "<|im_start|>user\n",
      "Say this is a test<|im_end|>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 测试tokenizer\n",
    "from transformers import AutoTokenizer,AutoModelForCausalLM\n",
    "# model = AutoModelForCausalLM.from_pretrained(\n",
    "#     \"./model/qwen_moe\",\n",
    "#     torch_dtype=\"auto\",\n",
    "#     device_map=\"auto\"\n",
    "# )\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"./pretrain_modify_from_TinyLlama/model/tokenizer_from_qwen_moe_chat\")\n",
    "print(\"bos token:\",tokenizer.bos_token)\n",
    "print(\"bos id\",tokenizer.bos_token_id)\n",
    "print(\"eos token:\",tokenizer.eos_token)\n",
    "print(\"eos id\",tokenizer.eos_token_id)\n",
    "print(\"pad token\",tokenizer.pad_token)\n",
    "print(\"pad_id:\", tokenizer.pad_token_id)\n",
    "print(\"vocab size\",tokenizer.vocab_size)\n",
    "print(len(tokenizer))\n",
    "messages=[\n",
    "        {\n",
    "            'role': 'user',\n",
    "            'content': 'Say this is a test',\n",
    "        }\n",
    "    ]\n",
    "res = tokenizer.apply_chat_template(messages)\n",
    "print(res[:3])\n",
    "print(tokenizer.decode(res))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [108386], 'attention_mask': [1]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(\"你好<|im_end|>\", add_special_tokens=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试不同版本的qwen模型tokenizer是否一致"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer,AutoModelForCausalLM\n",
    "tokenizer_qwen_1_chat = AutoTokenizer.from_pretrained(\"./model/qwen_1_1_8B_chat\",trust_remote_code=True)\n",
    "tokenizer_qwen_1_5_chat = AutoTokenizer.from_pretrained(\"./model/qwen_15_1_8B_chat\",trust_remote_code=True)\n",
    "tokenizer_qwen_moe = AutoTokenizer.from_pretrained(\"./model/qwen_moe\")\n",
    "tokenizer_qwen_moe_chat = AutoTokenizer.from_pretrained(\"./model/qwen_moe_chat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method QWenTokenizer.get_vocab of QWenTokenizer(name_or_path='./model/qwen_1_1_8B_chat', vocab_size=151851, model_max_length=8192, is_fast=False, padding_side='right', truncation_side='right', special_tokens={}, clean_up_tokenization_spaces=True),  added_tokens_decoder={\n",
      "\t\n",
      "}>\n",
      "<bound method PreTrainedTokenizerFast.get_vocab of Qwen2TokenizerFast(name_or_path='./model/qwen_15_1_8B_chat', vocab_size=151643, model_max_length=32768, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'eos_token': '<|im_end|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>']}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
      "\t151643: AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151644: AddedToken(\"<|im_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151645: AddedToken(\"<|im_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "}>\n",
      "<bound method PreTrainedTokenizerFast.get_vocab of Qwen2TokenizerFast(name_or_path='./model/qwen_moe', vocab_size=151643, model_max_length=32768, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'eos_token': '<|endoftext|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>']}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
      "\t151643: AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151644: AddedToken(\"<|im_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151645: AddedToken(\"<|im_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "}>\n",
      "<bound method PreTrainedTokenizerFast.get_vocab of Qwen2TokenizerFast(name_or_path='./model/qwen_moe_chat', vocab_size=151643, model_max_length=32768, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'eos_token': '<|im_end|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>']}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
      "\t151643: AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151644: AddedToken(\"<|im_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151645: AddedToken(\"<|im_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "}>\n",
      "<|endoftext|>\n",
      "<|im_end|>\n",
      " }>  }>  }>  }>\n",
      "绷 绷 绷 绷\n",
      " paciente  paciente  paciente  paciente\n",
      "共和国 共和国 共和国 共和国\n",
      "obbies obbies obbies obbies\n",
      "(Dictionary (Dictionary (Dictionary (Dictionary\n",
      "_INF _INF _INF _INF\n",
      " flashy  flashy  flashy  flashy\n",
      "느 느 느 느\n",
      "ethoven ethoven ethoven ethoven\n",
      "زان زان زان زان\n",
      " kin  kin  kin  kin\n",
      "رَ رَ رَ رَ\n",
      " Engineer  Engineer  Engineer  Engineer\n",
      "-turn -turn -turn -turn\n",
      "🤘 🤘 🤘 🤘\n",
      "� � � �\n",
      " FUNC  FUNC  FUNC  FUNC\n",
      " FileName  FileName  FileName  FileName\n",
      "干事 干事 干事 干事\n",
      "亳 亳 亳 亳\n",
      "跡 跡 跡 跡\n",
      " particle  particle  particle  particle\n",
      " Certification  Certification  Certification  Certification\n",
      "hog hog hog hog\n",
      " bev  bev  bev  bev\n",
      "李 李 李 李\n",
      "好感 好感 好感 好感\n",
      "煎 煎 煎 煎\n",
      "新型冠 新型冠 新型冠 新型冠\n",
      "ิต ิต ิต ิต\n",
      "人生的 人生的 人生的 人生的\n",
      "נט נט נט נט\n",
      "וצרים וצרים וצרים וצרים\n",
      " barring  barring  barring  barring\n",
      " Mega  Mega  Mega  Mega\n",
      "oplast oplast oplast oplast\n",
      "成人 成人 成人 成人\n",
      " Seeder  Seeder  Seeder  Seeder\n",
      "emás emás emás emás\n",
      "orgot orgot orgot orgot\n",
      " constrained  constrained  constrained  constrained\n",
      "Merchant Merchant Merchant Merchant\n",
      "midd midd midd midd\n",
      "狗狗 狗狗 狗狗 狗狗\n",
      "ι ι ι ι\n",
      "毅力 毅力 毅力 毅力\n",
      "ccb ccb ccb ccb\n",
      " Constraint  Constraint  Constraint  Constraint\n",
      "fung fung fung fung\n",
      " exped  exped  exped  exped\n",
      " Schn  Schn  Schn  Schn\n",
      "-alist -alist -alist -alist\n",
      " NSLog  NSLog  NSLog  NSLog\n",
      " Thumbnails  Thumbnails  Thumbnails  Thumbnails\n",
      "� � � �\n",
      " Cruise  Cruise  Cruise  Cruise\n",
      "例えば 例えば 例えば 例えば\n",
      "⌜ ⌜ ⌜ ⌜\n",
      "六个月 六个月 六个月 六个月\n",
      ".Meta .Meta .Meta .Meta\n",
      "� � � �\n",
      "Secret Secret Secret Secret\n",
      " Assumes  Assumes  Assumes  Assumes\n",
      " offre  offre  offre  offre\n",
      "friend friend friend friend\n",
      " broadcasts  broadcasts  broadcasts  broadcasts\n",
      "Bob Bob Bob Bob\n",
      "Legend Legend Legend Legend\n",
      " reinst  reinst  reinst  reinst\n",
      "Stra Stra Stra Stra\n",
      "ALS ALS ALS ALS\n",
      "_SAN _SAN _SAN _SAN\n",
      "屬於 屬於 屬於 屬於\n",
      "汭 汭 汭 汭\n",
      "𬺓 𬺓 𬺓 𬺓\n",
      "科室 科室 科室 科室\n",
      "URY URY URY URY\n",
      "毛孔 毛孔 毛孔 毛孔\n",
      " FY  FY  FY  FY\n",
      "repos repos repos repos\n",
      "overs overs overs overs\n",
      "理会 理会 理会 理会\n",
      "precedented precedented precedented precedented\n",
      ".z .z .z .z\n",
      " short  short  short  short\n",
      "𝕾 𝕾 𝕾 𝕾\n",
      " }},\n",
      "  }},\n",
      "  }},\n",
      "  }},\n",
      "\n",
      " tslint  tslint  tslint  tslint\n",
      "visão visão visão visão\n",
      " EX  EX  EX  EX\n",
      ".Hidden .Hidden .Hidden .Hidden\n",
      "งค์ งค์ งค์ งค์\n",
      "                                                                                                                                                                                                                                                                                           \n",
      "千 千 千 千\n",
      "⮳ ⮳ ⮳ ⮳\n",
      " resemble  resemble  resemble  resemble\n",
      " Tôi  Tôi  Tôi  Tôi\n",
      " organize  organize  organize  organize\n",
      "material material material material\n",
      "格 格 格 格\n",
      "/Footer /Footer /Footer /Footer\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer_qwen_1_chat.get_vocab)\n",
    "print(tokenizer_qwen_1_5_chat.get_vocab)\n",
    "print(tokenizer_qwen_moe.get_vocab)\n",
    "print(tokenizer_qwen_moe_chat.get_vocab)\n",
    "print(tokenizer_qwen_moe.eos_token)\n",
    "print(tokenizer_qwen_moe_chat.eos_token)\n",
    "counter = 0\n",
    "for _, moe_chat_token_id in tokenizer_qwen_moe_chat.get_vocab().items():\n",
    "    moe_chat_token = tokenizer_qwen_moe_chat.decode(moe_chat_token_id)\n",
    "    qwen1_chat_token = tokenizer_qwen_1_chat.decode(moe_chat_token_id)\n",
    "    qwen15_chat_token = tokenizer_qwen_1_5_chat.decode(moe_chat_token_id)\n",
    "    moe_base_token = tokenizer_qwen_moe.decode(moe_chat_token_id)\n",
    "    # if moe_chat_token!=qwen1_chat_token or moe_chat_token!=qwen15_chat_token or moe_chat_token!=moe_base_token:\n",
    "    # if moe_chat_token!=qwen1_chat_token:\n",
    "    print(qwen1_chat_token, qwen15_chat_token, moe_base_token, moe_chat_token)\n",
    "    if counter > 100:\n",
    "        break\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data =  [108636, 88970, 34187, 109654, 104447, 104283, 96050, 108264, 104162, 104283, 80158, 99684, 99425, 3837, 106359, 102104, 86119, 36885, 36885, 102993, 99590, 101132, 24968, 18493, 102508, 105465, 3837, 104432, 99980, 99689, 3837, 100636, 119356, 119454, 3837, 43288, 100132, 100438, 100838, 99572, 106708, 101107, 3837, 100431, 100157, 108464, 39907, 28311, 16, 5373, 30534, 42140, 104904, 8997, 97639, 47764, 38182, 101555, 116750, 26940, 99615, 99318, 108017, 25067, 15946, 99526, 3837, 99615, 99318, 108017, 105451, 46944, 111065, 104132, 102605, 3837, 99360, 111336, 102138, 99428, 99758, 18493, 111065, 39426, 17447, 3837, 111632, 11622, 99318, 119678, 119384, 99318, 106248, 111065, 69249, 3837, 99318, 45181, 99428, 100451, 106248, 68536, 99428, 108783, 100102, 1773, 108017, 104769, 36987, 42192, 42411, 3837, 220, 77288, 44934, 100089, 99079, 1773, 99486, 36587, 36987, 80443, 102657, 9909, 99687, 100279, 48272, 115826, 44934, 109267, 108809, 1773, 99615, 99318, 110830, 71268, 100272, 111994, 3837, 97639, 100390, 17340, 33126, 50511, 101265, 3837, 99290, 96808, 21287, 100084, 3837, 99486, 42140, 105009, 1773, 42140, 102108, 100676, 105001, 33108, 16628, 100623, 3837, 99915, 17340, 102446, 109284, 99953, 29732, 16530, 99851, 1773, 80443, 100165, 108424, 99486, 99312, 36587, 45629, 3837, 21287, 36407, 104435, 104795, 100438, 100838, 3837, 100165, 100132, 100798, 99555, 103976, 3837, 99306, 100268, 100532, 104355, 3837, 99306, 100423, 99793, 9370, 1773, 16530, 100798, 107834, 3837, 117227, 88970, 112609, 101036, 6313, 99999, 108443, 42140, 104904, 8997, 17, 5373, 115509, 52801, 100005, 104405, 8997, 101998, 104432, 3837, 80158, 99194, 102357, 3837, 105777, 100212, 99378, 104132, 117024, 3837, 99654, 73670, 100407, 100005, 104405, 3837, 105109, 104432, 98650, 3837, 73670, 105698, 105270, 1773, 101998, 105857, 105958, 26288, 101195, 9370, 3837, 35727, 103464, 100194, 100626, 18947, 100964, 99743, 99164, 101036, 1773, 99360, 99916, 101303, 31235, 100250, 105369, 3837, 101303, 104397, 108986, 3837, 62244, 104011, 99916, 80158, 82894, 108675, 18830, 88683, 34187, 1773, 31235, 100250, 105369, 101103, 104397, 104339, 34187, 3837, 113955, 111225, 101036, 1773, 101998, 99795, 102553, 3837, 102441, 105187, 34187, 3837, 104405, 99795, 80158, 100407, 34187, 1773, 100148, 59743, 109848, 106568, 27442, 17447, 3837, 30534, 100421, 99898, 3837, 99321, 99427, 99321, 26288, 99172, 3837, 99688, 29826, 108182, 109848, 34187, 1773, 101998, 18830, 99413, 3837, 104405, 99795, 101673, 115509, 34187, 8997, 18, 5373, 105916, 102435, 8997, 99688, 29826, 42140, 104125, 99283, 3837, 104219, 100424, 99376, 3837, 110913, 22243, 3837, 109944, 3837, 110985, 86119, 1773, 102033, 99688, 29826, 22045, 3837, 110239, 100815, 63109, 3837, 108015, 102050, 100034, 3837, 106217, 100677, 99283, 1773, 115037, 3837, 80158, 105745, 111225, 34187, 1773, 30534, 110280, 99283, 3837, 45181, 114554, 104487, 102435, 3837, 45181, 102435, 104487, 106556, 3837, 45181, 102340, 26939, 87267, 3837, 100211, 100409, 100005, 100353, 8997, 6667, 356, 8973, 101311, 99945, 104354, 5122, 105228, 100007, 101933, 99489, 94432, 6667, 220, 44991, 77540, 99437, 99945, 5122, 99945, 100044, 9370, 108841, 104055, 100007, 101928, 105641, 57218, 109930, 94432, 6667, 69372, 98749, 11622, 101431, 9370, 104085, 109782, 30709, 44991, 10904, 104710, 99878, 69905, 30709, 44991, 104339, 10958, 198, 6667, 58230, 237, 44991, 100007, 46670, 36556, 9370, 100963, 99452, 10904, 30709, 44991, 101906, 99459, 112224, 26232, 19108, 17447, 24156, 10958, 198, 100438, 100838, 99572, 3837, 26232, 101080, 105073, 3837, 66394, 112244, 28311, 17, 5373, 36605, 56568, 105321, 5122, 105622, 102142, 3837, 105622, 100949, 105153, 3837, 62922, 105715, 99283, 99461, 99392, 108902, 100854, 3837, 105715, 99283, 99461, 99392, 99730, 100854, 95053, 75882, 104672, 36993, 36407, 3837, 75882, 99314, 9370, 36993, 99314, 6313, 198, 105684, 100681, 100438, 100838, 99572, 13343, 3837, 80158, 99987, 99376, 106599, 8997, 18, 5373, 100356, 107530, 3837, 99623, 73670, 102155, 100064, 104166, 3837, 102435, 3837, 104961, 111684, 8997, 101283, 36407, 3837, 100148, 108363, 3837, 104325, 99222, 100047, 104432, 3837, 112760, 16530, 102435, 3837, 220, 115337, 42140, 105839, 100005, 100661, 3837, 101138, 100005, 101381, 8997, 73670, 42140, 50930, 101883, 86546, 17404, 47764, 3837, 109880, 104481, 90286, 3837, 45181, 100150, 17447, 102469, 105689, 108899, 3837, 105684, 101144, 102435, 103922, 3837, 108182, 87256, 105187, 99283, 104432, 34187, 8997, 102460, 3837, 42140, 57218, 17340, 102108, 3837, 99666, 101069, 3837, 100148, 105187, 3837, 99466, 100132, 2073, 110397, 107704, 104298, 110983, 33590, 71268, 99639, 100535, 17340, 3837, 73670, 101283, 36407, 3837, 60726, 33108, 102144, 100623, 102108, 104063, 3837, 109160, 114186, 57218, 109654, 101069, 104063, 3837, 49567, 101069, 104593, 3837, 117537, 73670, 104691, 26939, 26288, 104479, 15946, 3837, 108182, 105187, 99283, 100438, 100838, 99572, 3837, 104432, 104447, 88970, 17340, 34187, 8997, 18, 13, 35926, 65770, 3837, 99798, 44729, 104769, 36987, 106046, 8903, 44991, 65770, 106046, 95256, 3837, 107656, 100491, 105145, 101234, 99587, 11319, 57218, 99614, 38109, 105145, 21317, 99587, 11319, 41683, 16530, 99347, 99587, 11319, 99999, 3837, 100438, 100838, 9370, 100627, 29258, 18493, 99283, 8997, 100438, 100838, 99572, 100623, 105773, 103967, 101128, 17714, 101950, 111369, 1773, 18493, 101893, 104705, 3837, 97639, 101140, 99730, 30534, 100854, 99486, 101128, 103929, 101950, 8997, 101950, 20412, 106250, 100798, 33108, 39352, 9370, 1773, 101883, 87267, 32664, 104760, 17340, 100394, 101950, 103976, 104309, 111916, 106857, 101197, 106573, 104562, 1773, 99335, 100146, 105182, 99392, 104889, 99559, 102155, 100681, 110930, 101289, 99651, 1773, 77557, 3837, 101950, 87267, 57218, 103929, 99657, 5373, 101064, 5373, 99722, 5373, 102090, 102041, 5373, 99257, 5373, 116391, 105994, 99522, 101063, 1773, 101958, 99722, 117265, 39352, 101950, 1773, 56568, 110557, 101950, 101990, 32664, 103929, 102647, 99883, 20412, 115023, 9370, 1773, 101118, 103928, 105988, 31235, 104775, 5373, 108136, 5373, 106104, 101950, 9370, 99600, 3837, 101912, 33108, 99614, 57191, 104271, 100018, 104904, 57191, 111640, 3837, 49187, 101941, 3837, 105293, 3837, 100631, 33108, 99614, 57191, 108472, 100018, 106517, 105129, 8997, 105712, 3837, 16530, 99722, 104796, 116497, 9370, 83751, 59534, 68536, 99185, 3837, 101885, 104083, 101933, 1773, 100148, 103077, 102016, 101319, 1773, 91680, 100020, 99796, 91680, 101933, 101053, 101070, 1773, 104987, 52801, 107828, 1773, 97639, 102119, 113678, 104987, 100005, 85106, 1773, 103968, 3837, 104685, 105773, 102016, 56006, 97639, 111079, 110557, 9370, 102075, 99916, 1773, 101260, 99424, 101197, 103127, 107188, 3837, 105876, 100494, 20221, 20450, 8545, 102033, 100009, 100405, 103976, 3837, 101912, 57553, 105663, 52801, 90286, 100631, 49187, 56568, 112796, 101941, 1773, 88991, 104579, 3837, 103119, 105552, 3837, 42140, 111824, 3837, 71817, 18830, 104466, 9370, 100170, 99600, 1773, 104406, 114416, 57191, 104393, 1773, 108497, 100315, 44934, 1773, 104138, 104083, 69041, 101106, 101841, 102313, 119149, 112354, 103929, 99424, 33108, 86119, 3837, 77288, 104685, 16530, 99360, 99916, 104132, 101998, 105767, 106104, 101950, 1773, 100669, 101919, 99614, 33108, 45629, 103947, 100364, 73670, 100627, 56568, 104397, 101950, 104426, 8997, 102056, 100640, 104048, 101950, 102791, 3837, 105048, 110817, 107825, 45629, 112354, 3837, 42411, 73670, 108400, 105344, 39352, 101950, 33108, 101933, 16530, 99722, 104796, 8997, 53393, 104140, 104593, 109984, 1773, 102877, 26288, 99208, 26288, 99964, 102441, 105187, 99245, 120501, 120501, 99338, 88653, 34187, 3837, 31235, 103336, 100632, 101073, 3837, 104246, 102066, 8997, 36587, 100158, 97611, 100798, 96050, 35946, 99562, 99900, 100262, 3837, 102037, 113783, 3837, 102079, 106408, 99295, 99413, 69249, 100646, 102207, 1773, 35946, 100012, 77598, 64643, 77598, 30918, 99164, 99650, 104169, 113071, 99922, 85336, 104839, 106386, 100134, 75108, 99943, 1773, 99285, 34794, 104712, 38182, 8997, 101889, 102037, 99519, 102396, 22045, 41299, 50930, 99821, 31838, 1773, 112898, 102160, 99900, 107381, 80443, 100666, 3837, 104808, 102639, 99803, 3837, 106854, 105177, 50930, 99821, 31838, 1773, 99557, 103991, 100385, 9370, 117407, 99653, 100374, 1773, 50930, 99821, 31838, 103998, 35551, 99182, 72064, 18947, 99319, 8997, 104020, 107673, 3837, 80158, 101750, 100078, 104388, 1773, 18493, 102037, 104880, 103922, 3837, 99165, 15946, 36589, 104111, 104388, 64429, 99744, 72064, 34187, 1773, 42411, 99898, 29437, 100957, 35727, 35946, 104388, 1773, 99869, 99838, 1773, 91680, 100012, 60548, 100648, 103455, 8997, 104677, 50930, 99821, 31838, 101072, 3837, 102078, 18493, 104840, 100338, 42067, 51125, 105134, 31207, 3837, 74763, 111171, 3837, 99563, 101123, 3837, 103238, 16530, 97706, 8997, 101889, 99802, 7948, 21287, 99657, 3837, 102037, 73145, 100859, 100575, 104160, 104813, 105122, 35727, 35946, 99577, 101317, 51232, 99774, 102760, 8997, 101889, 99749, 7948, 102037, 73145, 100859, 100575, 111574, 3837, 85336, 104839, 99190, 105418, 101071, 1773, 101889, 101375, 104160, 8997, 88308, 99566, 104081, 102037, 99252, 101649, 8997, 88308, 113286, 3837, 105209, 1773, 99283, 99278, 14777, 102760, 1773, 118197, 103238, 101480, 8997, 107265, 100438, 100838, 3837, 35946, 33126, 99729, 11622, 104405, 100838, 116802, 14053, 46944, 18830, 104405, 100838, 100623, 3837, 101207, 101909, 100407, 102226, 107040, 100631, 101989, 3837, 74763, 105792, 100438, 100838, 100844, 1773, 100007, 101112, 100438, 100838, 3837, 99792, 101041, 9370, 71817, 104405, 106579, 118752, 36407, 49828, 18830, 104368, 39973, 100557, 33071, 14053, 16, 102875, 104405, 3837, 17, 81167, 104405, 101423, 101099, 9370, 104175, 3837, 18, 102124, 104405, 33108, 101099, 104336, 9909, 99464, 106628, 3837, 73670, 102064, 33108, 65676, 102064, 105446, 7552, 19, 93149, 104405, 105470, 57621, 1773, 62244, 105773, 101892, 16, 17, 18, 19, 101990, 71817, 104405, 106579, 118752, 3837, 100624, 100438, 100838, 58514, 100617, 104205, 198, 99363, 3837, 101048, 100418, 9370, 105552, 81217, 100170, 104904, 3837, 108015, 104619, 35926, 99332, 47534, 3837, 101099, 20412, 2073, 106846, 97907, 105728, 59217, 99802, 3837, 104016, 46944, 107255, 100160, 90395, 105699, 17714, 104282, 104016, 46944, 111095, 30709, 100160, 3837, 100356, 75117, 3837, 100147, 100178, 99486, 100438, 100838, 100423, 105080, 59217, 99749, 3837, 108630, 100005, 104405, 3837, 100359, 100005, 117070, 59217, 102232, 3837, 108630, 100005, 108452, 3837, 115226, 100005, 100848, 24968, 99236, 119328, 30918, 100005, 17340, 3837, 99236, 100047, 102256, 59217, 102320, 3837, 17340, 20412, 99328, 79256, 9370, 101239, 3837, 104205, 116391, 20412, 100438, 100838, 103099, 103517, 26381, 1773, 104038, 107352, 101421, 3837, 100356, 99331, 38953, 105594, 109715, 3837, 108630, 107693, 105910, 3837, 115226, 107693, 103134, 3837, 106266, 101891, 8997, 97639, 102563, 118966, 100218, 104774, 3837, 100659, 102215, 99424, 99998, 99257, 108080, 101928, 100694, 101950, 3837, 104205, 100438, 100838, 101909, 101986, 24360, 9370, 109457, 101139, 3837, 68536, 105479, 100438, 100838, 3837, 99520, 107421, 68536, 49828, 3837, 100141, 100438, 100838, 113317, 100623, 3837, 104560, 116420, 99917, 101090, 8997, 17340, 100002, 99283, 102828, 104140, 52853, 104012, 112929, 9370, 116629, 33108, 106904, 3837, 102056, 102185, 99937, 30709, 3837, 99518, 101930, 109141, 99719, 103148, 106628, 15946, 3837, 114858, 104928, 100694, 117597, 100268, 99349, 3837, 100624, 105684, 101928, 99283, 70927, 100798, 105565, 74040, 99535, 5373, 104424, 80158, 104892, 44934, 99336, 42192, 100173, 3837, 102085, 45181, 98749, 111798, 3837, 100137, 100034, 102004, 105998, 100438, 100838, 104250, 3837, 20412, 105125, 33447, 35727, 106187, 112376, 104009, 9370, 8997, 100438, 100838, 9370, 104250, 33108, 100659, 100423, 100798, 15946, 110091, 101882, 103943, 99164, 101552, 104612, 3837, 101912, 107740, 99307, 104433, 101072, 106266, 101882, 99285, 3837, 64689, 64689, 115705, 100671, 81217, 33983, 99364, 52801, 96050, 103985, 9370, 32, 33108, 105172, 33, 109487, 15946, 102099, 50404, 80443, 104036, 104196, 33, 3837, 111151, 106266, 99883, 18830, 104529, 105679, 109487, 3837, 102351, 99314, 100194, 104311, 104392, 99683, 99208, 99683, 52510, 3837, 104979, 56006, 40916, 100820, 17340, 104250, 34187, 102075, 17447, 41299, 63109, 33108, 100638, 104969, 107152, 100034, 1773, 100383, 101913, 102956, 119443, 96050, 109516, 17447, 103930, 56568, 101068, 104904, 100005, 100438, 100838, 8997, 103945, 100627, 100005, 100438, 100838, 3837, 100000, 18830, 39907, 9370, 8997, 100692, 100005, 104116, 96050, 99257, 5373, 100134, 5373, 116823, 49567, 102159, 101958, 100005, 112526, 3837, 100720, 100005, 102004, 3837, 104775, 100689, 100005, 100661, 3837, 20412, 100641, 35926, 101381, 108060, 1773, 100151, 99992, 101492, 107932, 100669, 104036, 9370, 35926, 101381, 3837, 104311, 32664, 104782, 9370, 104036, 101144, 116279, 3837, 101966, 9370, 104243, 100006, 104904, 100005, 101135, 8997, 99802, 64682, 99486, 106614, 102050, 3837, 99424, 101047, 108591, 100798, 100132, 99349, 26232, 103562, 106187, 3837, 57191, 19108, 57191, 22045, 3837, 101278, 45181, 110396, 100034, 15946, 102050, 20221, 111186, 99837, 22226, 3837, 100002, 104782, 110717, 48921, 101412, 103976, 3837, 103977, 104219, 18830, 99413, 3837, 23384, 101490, 16530, 100397, 8997, 99749, 64682, 20412, 101278, 68862, 99701, 3837, 56568, 87267, 100151, 101144, 100034, 5373, 99361, 3837, 77288, 104605, 29826, 29826, 110234, 96050, 101066, 106735, 111954, 22045, 102072, 15946, 3837, 26232, 105053, 100005, 104753, 3837, 68862, 99701, 99645, 99376, 3837, 100000, 32664, 104049, 100438, 100838, 9370, 104904, 33108, 100341, 8997, 104718, 100415, 100793, 101312, 101725, 3837, 101885, 104468, 101112, 99283, 100565, 33108, 103059, 3837, 111898, 99283, 102154, 99350, 3837, 73670, 92133, 101883, 99283, 99792, 99729, 9370, 99600, 3837, 29524, 100646, 77959, 21515, 5373, 58405, 99329, 5373, 117933, 5373, 111261, 57191, 99729, 102057, 99366, 49567, 3837, 104468, 71817, 101883, 99534, 99863, 9370, 102275, 3837, 99654, 73670, 99738, 99283, 102357, 104838, 100819, 3837, 101982, 73670, 101933, 100005, 106479, 8997, 80443, 99880, 108182, 106586, 3837, 68536, 99880, 108744, 103939, 106586, 108744, 1773, 107427, 100372, 104220, 3837, 73670, 99340, 100198, 101134, 103929, 106076, 25511, 90395, 32664, 104394, 105369, 104133, 100438, 101077, 1773, 39165, 59151, 100672, 101711, 13343, 3837, 100150, 101950, 36993, 30709, 107048, 1773, 46944, 99222, 100964, 100160, 109454, 99283, 101950, 3837, 39165, 115770, 101068, 104658, 13343, 3837, 103939, 100394, 117911, 98650, 8997, 101045, 3837, 100438, 100838, 64471, 52801, 3837, 108764, 101928, 102334, 101950, 104248, 3837, 73670, 104015, 29490, 101921, 52801, 100005, 106479, 1773, 101099, 44091, 104625, 105675, 100359, 9370, 3837, 100645, 102062, 99283, 102075, 99666, 9370, 108574, 90395, 88991, 102086, 100005, 100021, 3837, 114116, 103928, 105698, 104432, 33108, 107115, 103976, 90395, 108739, 71817, 106268, 8997, 105684, 101928, 101950, 13343, 3837, 73670, 109733, 100005, 106479, 3837, 99190, 28726, 29826, 57191, 53153, 101929, 103929, 106076, 3837, 110239, 109966, 69516, 100224, 99283, 1773, 106124, 3837, 99666, 104125, 99283, 3837, 115226, 107290, 101919, 101106, 3837, 104049, 112339, 100000, 73670, 9370, 3837, 101194, 39907, 36407, 104463, 99283, 3837, 29524, 105876, 99565, 104191, 105950, 5373, 99190, 112498, 103976, 8997, 62244, 107967, 100367, 104334, 3837, 99652, 104077, 70790, 112137, 100037, 34187, 3837, 107427, 108720, 3837, 101109, 101938, 99250, 105214, 100373, 1773, 102056, 23031, 101053, 100037, 100875, 107064, 101990, 99190, 105772, 3837, 104435, 108025, 101135, 60548, 99652, 1773, 110504, 99190, 99569, 108008, 3837, 30440, 60726, 99190, 31235, 100250, 9370, 104868, 3837, 102033, 100672, 22045, 3837, 104871, 105698, 99204, 102262, 111369, 198, 106279, 3837, 100438, 100838, 33108, 99420, 101950, 99788, 102513, 100250, 3837, 108747, 99283, 107126, 105413, 99719, 100682, 104426, 3837, 68536, 67338, 70589, 39907, 3837, 73670, 107980, 105053, 52801, 100005, 100438, 1773, 39165, 100438, 100838, 33108, 99420, 99451, 99788, 104496, 100627, 33447, 3837, 102003, 101107, 49828, 102435, 90395, 111025, 36407, 100366, 104394, 104820, 1773, 151645]\n",
    "print()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 检查tinyllama bin数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 检查raw wanjuan数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "161\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-003725-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-003277-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-007090-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_Patent-cn_part-008323-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-004844-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-006809-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-000122-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-003903-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-010669-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-004712-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-005091-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-008609-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-007224-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-005491-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_Patent-cn_part-003260-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-002169-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-009966-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-003892-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-004764-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-010205-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-000467-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-009261-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-008840-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-000895-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-000520-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_Exam-cn_part-003756-a894b46e.jsonl\n",
      "dict_keys(['id', 'q_type', 'q_main', 'option_a', 'option_b', 'option_c', 'option_d', 'option_e', 'std_ans', 'answer', 'answer_detail', 'grade', 'major', 'keypoint'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-010597-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-010550-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_ChinaNews-cn_part-006853-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-008218-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-003827-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-003716-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-001819-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-005408-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-005115-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-004796-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_ChinaNews-cn_part-008323-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-007501-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-001772-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_Law-cn_part-009942-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-001199-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-003508-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-009734-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-009258-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_Law-cn_part-001398-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_Law-cn_part-007119-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-007261-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_Law-cn_part-001744-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-000036-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-007566-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-001835-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-010223-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-005138-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-002228-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-002619-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_zh/CN_WebText-cn_part-003516-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007783-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007604-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004245-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-002100-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010191-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007994-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004104-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-003563-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004474-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-005912-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-005916-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008580-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007837-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-002845-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000369-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-001238-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000623-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-005013-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000790-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000419-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-001597-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008830-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008279-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-003238-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010631-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-009861-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-005850-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008012-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008979-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-002262-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006313-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010015-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-001017-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008056-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004964-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000341-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-003752-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007380-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006801-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004392-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004736-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000442-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000286-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-005354-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006822-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007516-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008343-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008552-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007020-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008443-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006009-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006673-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010518-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007037-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-001031-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010423-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010119-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-002055-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-009210-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-003751-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010490-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-005561-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007915-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-009967-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004293-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-001456-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000363-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-002387-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007248-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000376-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008973-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-003273-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-009804-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-009427-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-008163-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-005055-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-001966-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-003201-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010417-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006726-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000065-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010441-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-010594-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006180-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-009538-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004584-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006621-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004085-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007521-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000655-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-003067-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-002990-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006656-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000346-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-007655-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-005034-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004580-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000020-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-006674-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-001306-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-003835-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-009759-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004923-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-004004-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n",
      "/data/step2_data_cleaning/wanjuan/wanjuan_en/EN_WebText-en_part-000841-a894b46e.jsonl\n",
      "dict_keys(['id', 'content'])\n"
     ]
    }
   ],
   "source": [
    "import glob \n",
    "import json\n",
    "dir_list = glob.glob(\"/data/step2_data_cleaning/wanjuan/**/*.jsonl\", recursive=True)\n",
    "print(len(dir_list))\n",
    "for file_dir in dir_list:\n",
    "    with open(file_dir, encoding=\"utf-8\") as f:\n",
    "        for row in f:\n",
    "            text = json.loads(row)\n",
    "            print(file_dir)\n",
    "            print(text.keys())\n",
    "            break "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "map(lambda char: 1 if char.isalpha() else 0, sample[self.text_key])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "packeddata 1 epoch try"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3]\n",
      "[3, 4, 5, 6]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "n_chunk = 4\n",
    "idx = 0\n",
    "L = [0,1,2,3,4,5,6]\n",
    "end = False\n",
    "for i in range(10):\n",
    "    if n_chunk > len(L[idx:]):\n",
    "        if  L[idx:]!=0 and end==False:\n",
    "            idx = len(L)-n_chunk\n",
    "            end = True\n",
    "        else:\n",
    "            idx = 0\n",
    "            end = False\n",
    "    print(L[idx: idx+n_chunk])\n",
    "    idx += n_chunk\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# huggingface save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The model is automatically converting to bf16 for faster inference. If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\".\n",
      "Try importing flash-attention for faster inference...\n",
      "Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "QWenLMHeadModel(\n",
      "  (transformer): QWenModel(\n",
      "    (wte): Embedding(151936, 2048)\n",
      "    (drop): Dropout(p=0.0, inplace=False)\n",
      "    (rotary_emb): RotaryEmbedding()\n",
      "    (h): ModuleList(\n",
      "      (0-23): 24 x QWenBlock(\n",
      "        (ln_1): RMSNorm()\n",
      "        (attn): QWenAttention(\n",
      "          (c_attn): Linear(in_features=2048, out_features=6144, bias=True)\n",
      "          (c_proj): Linear(in_features=2048, out_features=2048, bias=False)\n",
      "          (core_attention_flash): FlashSelfAttention()\n",
      "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (ln_2): RMSNorm()\n",
      "        (mlp): QWenMLP(\n",
      "          (w1): Linear(in_features=2048, out_features=5504, bias=False)\n",
      "          (w2): Linear(in_features=2048, out_features=5504, bias=False)\n",
      "          (c_proj): Linear(in_features=5504, out_features=2048, bias=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): RMSNorm()\n",
      "  )\n",
      "  (lm_head): Linear(in_features=2048, out_features=151936, bias=False)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from transformers import AutoConfig\n",
    "model_path = \"***\"\n",
    "sys.path.append(model_path)\n",
    "from modeling_qwen import QWenLMHeadModel, QWenBlock\n",
    "config = AutoConfig.from_pretrained(model_path,trust_remote_code=True)\n",
    "model = QWenLMHeadModel(config)\n",
    "print(model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 检查数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.int32'> 419635200\n",
      "[139 132   1 ... 115   0   0] 1678540800\n",
      "<memory at 0x7fef4c8568c0>\n",
      "[ 99467 104897     11  16530  41406  64643  34794 100889  77419  16530]\n",
      "[104897     11  16530  41406  64643  34794 100889  77419  16530  14880]\n",
      "len:  2049 2775 4\n",
      "青霞,不料司空摘星不请自到,薛冰也黑衣蒙面而来.\n",
      "放走江青霞,一个叫\"红鞋子\"的组织浮出水面.有蛇王拿出的平南王府地形图,陆小凤夜入平南王府,不想与金九龄遭遇,差点让金九龄的一剑要了性命.在蛇王的安排下,陆小凤终于揭开公孙大娘神秘的面纱,并在薛冰神秘失踪后,将制服的公孙大娘缉拿归案.\n",
      "原来,\"公孙大娘\"并不是别人,而是薛冰所扮,送\"公孙大娘\"归案,引金九龄现身,乃陆小凤一手策划.\n",
      "谁知从案发后司空摘星偷取牡丹红帕开始,陆小凤就开始怀疑金九龄.尽管金九龄得\"易水歌\"春秋剑谱密笈,但在陆小凤面前,金九龄只有束手待毙.决战前后剑神西门吹雪与白云城主叶孤城相约在\"月圆之夜,紫禁之巅\"进行决战,消息传出四海皆惊,江湖人士齐聚京城欲睹峥嵘.不久,大侠李雁北遭到暗算,龟孙大爷被人灭口,红颜知己欧阳晴也身中剧毒,而陆小凤追查凶手却毫无头绪,但是他预感到\"紫禁一战\"的背后一定暗藏玄机.月圆之夜,紫禁之巅,西门吹雪与叶孤城双双亮剑,所有人都屏住呼吸等候着这场旷世决斗的开始,突然,陆小凤在叶孤城的宝剑上看出了所有问题的症结所在......银钩赌坊西方魔教玉罗刹教主之子玉天宝被杀,银钩赌坊主人蓝胡子栽赃陆小凤.\n",
      "陆小凤原本不想管此等闲事,但西方魔教两大护法孤松,枯竹找上门来,陆小凤也就林志颖版陆小凤(4张)答应下来.\n",
      "但即便如此,陆小凤依然在与黑虎唐旗下黄犬堂主聂小全的较量中,拿到罗刹牌.\n",
      "\n",
      "\n",
      "原形毕现的蓝胡子丧命于寻仇的蜀中银鹞子方玉飞之女香香的\"食心虫\",暗中勾结的苦竹被苦心孤诣设计此局清理门户的玉罗刹所杀.幽灵山庄陆小凤因染指西门吹雪的妻子遭西门吹雪追杀.为避免朋友相残,陆小凤由勾魂使者引见藏身于人间鲜有耳闻的幽灵山庄.\n",
      "老刀把子谋划已久\"天雷行动\"即将开始,幽灵山庄全体成员被老刀把子兵分三路向武当秘密进发.四月十三日,江湖人士齐聚武当参加武当册封新掌门人大典日子,老刀把子制定\"天雷行动\",想利用这个机会杀死武当掌门人石雁,长老木道人,苦戒大师,鹰眼老七四大仇人,夺取藏匿于石雁头顶紫金冠里的一本记录江湖人士隐秘帐簿.大典当日江湖人士齐聚武当,老刀把子周密计划,陆小凤顺利得手大功告成以酒相庆,陆小凤突然撕下老刀把子伪装.\n",
      "原来获取木道人七星剑才\"天雷行动\"真正目,\"勾魂使者\"乔装老刀把子丧命木道人剑下,谁是真正的老刀把子?带着这个疑问陆小凤重返幽灵山庄,从刚刚遭受杀戮的幽灵山庄救出叶灵.此时一缕剑穗让陆小凤找到涂炭幽灵山庄线索.赶回武当,适逢武当掌门人石雁暴病而亡,资历最深的木道人出任新掌门.册封时陆小凤当众揭开木道人就是老刀把子的真相,使一个藏了十五年秘密浮出水面.\n",
      "事已至此,真相毕露,木道人被亲生女儿叶雪一剑了断,验证了邪恶有界报应有天古训.剑神一笑剑神西门吹雪请陆小凤出手相助,陆小凤奇怪天下难道还有西门吹雪办不了的事?不久,陆小凤造访鱼龙混杂的大漠边陲小镇,而西门吹雪让他找的人 -  - 巴山剑派大师兄柳乘风竟然已经被人一剑封喉.陆小凤发现这座黄石小镇竟是龙潭虎穴,破落的大眼客栈里也是杀机四伏.陆小凤顺藤摸瓜发现命案与小镇中一位被贬的王妃宫素素有牵连,而宫素素却对他情有独钟.几经周折,陆小凤终于在荒无人烟的沙漠中发现了秘密宝藏,与此同时,小镇上的黑手正在密谋除掉陆小凤......凤舞九天江湖告急,太平王之女玉屏公主突遭绑架,王爷震怒,黑白两道人人自危......血衣之谜武林秘笈重现江湖,神秘人假扮陆小凤夜入皇宫夺走\"星邪剑谱\".一时间,陆小凤成了武林公敌朝廷要犯,\"血衣堂\"借机要挟陆小凤加入组织,原来真的\"星邪剑谱\"还在皇宫大内,需要陆小凤与六位血衣童子联手才能盗出真剑谱.陆小凤走投无路只得\"从命\",其实他早与南平郡王定下计策欲借此机会铲除武林毒瘤\"血衣堂\".几经周折,陆小凤终于盗出真剑谱,突然形式陡转直下......\n",
      "必须说明的是,在古龙的原作中,只有以下7部:金鹏王朝, 绣花大盗,决战前后,银钩赌坊,幽灵山庄,凤舞九天,剑神一笑.其余三部只有电影.歌曲:CCTV版歌曲名:双侠\n",
      "专辑名:陆小凤片尾曲 \n",
      "作 词:崔恕\n",
      "作 曲:无级生(maxam)\n",
      "编 曲:谭伊哲\n",
      "歌 手:无极生\n",
      "鸟一对\n",
      "天空海阔分飞\n",
      "酒一杯\n",
      "各自天南地北\n",
      "两双腿\n",
      "踏着时间去追一个完美\n",
      "不后悔\n",
      "我们一去不回\n",
      "你是谁\n",
      "沾染日月清辉\n",
      "我是谁喝过银河之水\n",
      "趁酒醉双双到人间度一场是非\n",
      "下一回七度空间相会\n",
      "rap\n",
      "我跟你相遇中国遥遥十万八千里\n",
      "我跟你相遇在不属于自己的土地\n",
      "是缘分是巧合让你我成为兄弟\n",
      "是可遇不可求那种无形的默契\n",
      "鸟一对在天空中相会\n",
      "酒一杯冲淡是是非非\n",
      "两双腿踏过千山万水从来不累\n",
      "纵不回也在他乡交汇\n",
      "你是谁为我插刀两肋\n",
      "我是谁对你掏心掏肺\n",
      "风一吹世间的传说转眼就破碎\n",
      "不后退我们尽力而为\n",
      "手一挥掌一推\n",
      "浩瀚宇宙没有极限的范围\n",
      "小天地大智慧\n",
      "无极世界其实在你我心扉\n",
      "\n",
      "清静心灵没有极限的范围\n",
      "\n",
      "无极世界其实没什么绝对\n",
      "已经忘记与你一齐经历过了多少的血和泪\n",
      "这种感情已经超越过了时间超越了年和岁\n",
      "面对任何眼泪受过多少的罪我们也不撤退\n",
      "一齐来吧喝醉装满烈酒的杯怎么碰不会碎\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "经得起风得起浪才知道什么叫珍贵\n",
      "同尝过甘吃过苦才明白什么叫无悔\n",
      "天地之大融会无极两端相遇一对\n",
      "转眼间就会出现人间光芒之最林志颖版为了他 \n",
      "演唱者:陶红 \n",
      "飞沙中寒风中\n",
      "故人去流浪\n",
      "在远方\n",
      "他的声音就是我梦想\n",
      "当每次夜静时\n",
      "思念又回荡\n",
      "笛声苍凉他可无恙\n",
      "有了他的地方才是我的天堂\n",
      "我心随风飘扬落在他的身旁\n",
      "为了他甘愿告别那故乡的蓝天\n",
      "为了他甘愿告别那笛声的悠扬\n",
      "穿越过千山万水\n",
      "我会向他奔去\n",
      "只为能跟他去流浪流浪四方新九品芝麻官:张默 饰 陆小凤  张默饰演陆小凤人物性格:天性率直,思想单纯,外表是叛逆小子,内心却有着对正义的坚持,对公理的执着,最大的梦想就是做个像\"包青天\"一样的好官,为百姓主持公道,经过世事的磨砺,终于达成自己的梦想.但在感情方面却是个迟钝的人,面对两个红颜知己,混乱不清,叫人既爱且恨.\n",
      "全集在线观看:.vedio-content{background:#fafafb; 0}.vedio-content ul{margin:0 3px; ul li{ 6px; ul li a{ ul li .pic-show{ solid #ccc}.vedio-content ul li . solid #45a909}.vedio-content ul li img{ a{ a:hover{ .v-icon{ url(); html .vedio-content .v-icon{ .new-icon{ url() no-repeat; html .new-icon{ .drama-tab{margin\n"
     ]
    }
   ],
   "source": [
    "HDR_MAGIC = b\"LITPKDS\"\n",
    "HDR_SIZE = 24\n",
    "import struct\n",
    "import numpy as np\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "dtypes = {1: np.uint8, 2: np.int8, 3: np.int16, 4: np.int32, 5: np.int64, 6: np.float32, 7: np.float64, 8: np.uint16}\n",
    "def read_header(path):\n",
    "    with open(path, \"rb\") as f:\n",
    "        magic = f.read(len(HDR_MAGIC))\n",
    "        assert magic == HDR_MAGIC, \"File doesn't match expected format.\"\n",
    "        version = struct.unpack(\"<Q\", f.read(8))\n",
    "        assert version == (1,)\n",
    "        (dtype_code,) = struct.unpack(\"<B\", f.read(1))\n",
    "        dtype = dtypes[dtype_code]\n",
    "        (chunk_size,) = struct.unpack(\"<Q\", f.read(8))\n",
    "    return dtype, chunk_size\n",
    "\n",
    "\n",
    "bin_data_dir = \"/data1/step3_final_data/chat/chat_process0_0000000003.bin\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"/home/calfa100/gqs/Steel-LLM/pretrain_modify_from_TinyLlama/model/tokenizer_from_qwen_moe_chat\")\n",
    "dtype, chunk_size = read_header(bin_data_dir)\n",
    "print(dtype, chunk_size)\n",
    "mmap = np.memmap(bin_data_dir, mode=\"r\", order=\"C\", offset=HDR_SIZE)\n",
    "print(mmap, len(mmap))\n",
    "view = memoryview(mmap)\n",
    "print(view)\n",
    "arr1 = np.frombuffer(view, dtype=dtype, count=2049, offset=0)\n",
    "# 偏移量需要按字节来\n",
    "arr2 = np.frombuffer(view, dtype=dtype, count=2049, offset=np.dtype(dtype).itemsize*1)\n",
    "print(arr1[:10])\n",
    "print(arr2[:10])\n",
    "text = tokenizer.decode(arr1)\n",
    "print(\"len: \", len(arr1), len(text), np.dtype(dtype).itemsize)\n",
    "print(text[:20000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
